#!/usr/bin/env python
# -*- coding: utf-8 -*-
# 获取规则sql执行，比较数据并发送邮件
# author:sunlyliuh@163.com
import os
import datetime
import sys
from io import StringIO
import contextlib
from common_quality import common_quality,execute_sql
from common.DbMySQL import DbMySQL
from common.tools import getDBConfig,sendmail

def getDbCloud():
    cloud_37 = getDBConfig('metadata')
    db_colud = DbMySQL(cloud_37['host'], cloud_37['user'], cloud_37['passwd'], cloud_37['db'])
    return db_colud

@contextlib.contextmanager
def stdoutIO(stdout=None):
    old = sys.stdout
    if stdout is None:
        stdout = StringIO.StringIO()
    sys.stdout = stdout
    yield stdout
    sys.stdout = old

def insertRuleToTable(rule):
    """
    插入规则到表中
    :return:
    """

    quality = common_quality(rule_type=rule['rule_type'], db_name=rule['db_name'], table_name=rule['table_name'], \
                             column_name=rule['column_name'], custom_sql=rule['custom_sql'],
                             sampling_mode=rule['sampling_mode'], filter_condition=rule['filter_condition'])
    executeSql = quality.get_execute_sql()
    print(executeSql)

    # 判断是否已经存在未执行的规则，已经存在的话就不加入了
    sql = '''
                    select id,rule_id from hive_rule_execute_log where rule_id='%s' and date(create_time)='%s'
                ''' % (rule['id'], datetime.datetime.now().strftime('%Y-%m-%d'))
    existRow = db_colud.queryRow(sql)
    if existRow:
        print('rule_id:%s,status=0 is exist' % (rule['id']))
    else:
        insertDict = {
            'rule_id': rule['id'],
            'status': 0,
            'error_msg': '',
            'start_time': '1970-01-01 00:00:00',
            'end_time': '1970-01-01 00:00:00',
            'execute_seconds': 0,
            'execute_sql': executeSql,
            'create_time': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        }

        db_colud.insert('hive_rule_execute_log', insertDict)


def addRuleToExecute():
    """
    将规则加入到执行列表中
    """
    executeDate = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')

    # 添加逻辑 判断makefile 文件是否已经生成，生成了对应文件才加入执行
    ruleSql = '''
    select r.* from hive_quality_rule r 
    inner join hive_schedule_status s on r.table_name=s.table_name
    where r.status='on' and r.execute_type=1 and s.status='SUCCEEDED' and s.execute_date='%s'
    ''' % (executeDate)
    ruleAll = db_colud.queryAll(ruleSql)

    if ruleAll:
        for rule in ruleAll:
            insertRuleToTable(rule)
    else:
        print('没有需要添加的规则')


def getCompareData(ruleId,modeType):
    """
    获取要对比的数据
    """
    if modeType == '2':
        startDate = (datetime.datetime.now()-datetime.timedelta(days=2)).strftime('%Y-%m-%d')
        endDate = (datetime.datetime.now()-datetime.timedelta(days=2)).strftime('%Y-%m-%d')
    elif modeType == '3':
        startDate = (datetime.datetime.now()-datetime.timedelta(days=8)).strftime('%Y-%m-%d')
        endDate = (datetime.datetime.now()-datetime.timedelta(days=2)).strftime('%Y-%m-%d')
    elif modeType == '4':
        startDate = (datetime.datetime.now()-datetime.timedelta(days=31)).strftime('%Y-%m-%d')
        endDate = (datetime.datetime.now()-datetime.timedelta(days=2)).strftime('%Y-%m-%d')
    sql = '''
    select ifnull(avg(result_num),0) result_num
    from hive_rule_execute_num
    where calculate_date >='%s' and calculate_date<='%s' and rule_id='%s'
    ''' % (startDate,endDate,ruleId)
    resultNum = db_colud.queryRow(sql)

    return resultNum

def getCompareMode(compareMode):
    """
    获取比较的表达式做解析
    """
    compareStr = ''
    if compareMode == 'gt':
        compareStr = '>'
    elif compareMode == 'gte':
        compareStr = '>='
    elif compareMode == 'lt':
        compareStr = '<'
    elif compareMode == 'lte':
        compareStr = '<='
    elif compareMode == 'equal':
        compareStr = '=='
    elif compareMode == 'nequal':
        compareStr = '!='
    elif compareMode == 'rise':
        compareStr = '>='
    elif compareMode == 'decline':
        compareStr = '<='
    else:
        compareStr = 'error'

    return compareStr

def getRuleTypeName(ruleType):
    """
    获取规则类型名称
    """
    typeName= ''
    if ruleType == '1':
        typeName = '表级规则'
    elif ruleType == '2':
        typeName = '字段级规则'
    else:
        typeName = '自定义sql'

    return typeName


def getCheckTypeName(checkType):
    """
    获取校验类型
    """
    checkTypeName = ''

    if checkType == '1':
        checkTypeName = '数值型'
    else:
        checkTypeName = '波动率型'

    return checkTypeName


def getCheckModeName(checkMode):
    """
    获取校验方式名称
    """
    checkModeName = ''
    if checkMode == '1':
        checkModeName = '与固定值对比'
    elif checkMode == '2':
        checkModeName = '与昨天对比'
    elif checkMode == '3':
        checkModeName = '7天平均值对比'
    else:
        checkModeName = '30天平均值对比'

    return checkModeName


def getCompareModeName(compareMode):
    """
    获取比较方式
    """
    if compareMode == 'rise':
        compareModeName = '上升'
    elif compareMode == 'decline':
        compareModeName = '下降'
    elif compareMode == 'abs':
        compareModeName = '绝对值'
    elif compareMode == 'gt':
        compareModeName = '>'
    elif compareMode == 'gte':
        compareModeName = '>='
    elif compareMode == 'lt':
        compareModeName = '<'
    elif compareMode == 'lte':
        compareModeName = '<='
    elif compareMode == 'equal':
        compareModeName = '='
    elif compareMode == 'nequal':
        compareModeName = '!='
    else:
        compareModeName = ''

    return compareModeName


def sendWarningMail(ruleInfo,result_run,compare_run=0):
    """
    发送警告邮件
    """
    mail = 'mail'
    subject = '数据质量监控'
    receivers = ruleInfo['warning_receiver'].strip(';').split(';')

    compareTitle = ''
    compareValue = ''
    if ruleInfo['check_type'] == '1':
        checkTypeTitle = '期望值'
        checkTypeValue = ruleInfo['excepted_value']
    else:
        checkTypeTitle = '波动值'
        checkTypeValue = ruleInfo['percent_min']+'%'

    messages = '<html>各位好，<br />告警信息如下：<br/>'
    messages += '<table border="1" style="border-collapse:collapse;text-align:center;">'
    if ruleInfo['rule_type'] == '1':
        if ruleInfo['check_type'] == '1' :
            messages += '<tr><td>规则名称</td><td>规则类型</td><td>数据库名</td><td>表名</td><td>校验类型</td><td>校验方式</td><td>比较方式</td><td>'+checkTypeTitle+'</td><td>今日数据</td></tr>'
            messages += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \
            (ruleInfo['rule_name'],getRuleTypeName(ruleInfo['rule_type']),ruleInfo['db_name'],ruleInfo['table_name'],getCheckTypeName(ruleInfo['check_type']),getCheckModeName(ruleInfo['check_mode']),getCompareModeName(ruleInfo['compare_mode']),checkTypeValue,result_run)
        else:
            messages += '<tr><td>规则名称</td><td>规则类型</td><td>数据库名</td><td>表名</td><td>校验类型</td><td>校验方式</td><td>比较方式</td><td>'+checkTypeTitle+'</td><td>今日数据</td><td>对比数据</td></tr>'
            messages += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \
            (ruleInfo['rule_name'],getRuleTypeName(ruleInfo['rule_type']),ruleInfo['db_name'],ruleInfo['table_name'],getCheckTypeName(ruleInfo['check_type']),getCheckModeName(ruleInfo['check_mode']),getCompareModeName(ruleInfo['compare_mode']),checkTypeValue,result_run,compare_run)
    elif ruleInfo['rule_type'] == '2':
        if ruleInfo['check_type'] == '1':
            messages += '<tr><td>规则名称</td><td>规则类型</td><td>数据库名</td><td>表名</td><td>字段名</td><td>校验类型</td><td>校验方式</td><td>比较方式</td><td>'+checkTypeTitle+'</td><td>今日数据</td></tr>'
            messages += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \
            (ruleInfo['rule_name'],getRuleTypeName(ruleInfo['rule_type']),ruleInfo['db_name'],ruleInfo['table_name'],ruleInfo['column_name'],getCheckTypeName(ruleInfo['check_type']),getCheckModeName(ruleInfo['check_mode']),getCompareModeName(ruleInfo['compare_mode']),checkTypeValue,result_run)
        else:
            messages += '<tr><td>规则名称</td><td>规则类型</td><td>数据库名</td><td>表名</td><td>字段名</td><td>校验类型</td><td>校验方式</td><td>比较方式</td><td>'+checkTypeTitle+'</td><td>今日数据</td><td>对比数据</td></tr>'
            messages += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \
            (ruleInfo['rule_name'],getRuleTypeName(ruleInfo['rule_type']),ruleInfo['db_name'],ruleInfo['table_name'],ruleInfo['column_name'],getCheckTypeName(ruleInfo['check_type']),getCheckModeName(ruleInfo['check_mode']),getCompareModeName(ruleInfo['compare_mode']),checkTypeValue,result_run,compare_run)
    else:
        if ruleInfo['check_type'] == '1':
            messages += '<tr><td>规则名称</td><td>规则类型</td><td>SQL</td><td>校验类型</td><td>校验方式</td><td>比较方式</td><td>'+checkTypeTitle+'</td><td>今日数据</td></tr>'
            messages += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \
            (ruleInfo['rule_name'],getRuleTypeName(ruleInfo['rule_type']),ruleInfo['sql'], getCheckTypeName(ruleInfo['check_type']), getCheckModeName(ruleInfo['check_mode']),
                     getCompareModeName(ruleInfo['compare_mode']), checkTypeValue, result_run)
        else:
            messages += '<tr><td>规则名称</td><td>规则类型</td><td>SQL</td><td>校验类型</td><td>校验方式</td><td>比较方式</td><td>'+checkTypeTitle+'</td><td>今日数据</td><td>对比数据</td></tr>'
            messages += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \
            (ruleInfo['rule_name'],getRuleTypeName(ruleInfo['rule_type']),ruleInfo['sql'], getCheckTypeName(ruleInfo['check_type']), getCheckModeName(ruleInfo['check_mode']),
                     getCompareModeName(ruleInfo['compare_mode']), checkTypeValue, result_run,compare_run)


    messages += '</table></html>'

    sendmail(mail,subject,receivers,messages)

def sendWarningMailAll(ruleInfoAll):
    """
    发送警告邮件
    """
    mail = 'mail'
    subject = '数据质量监控'
    receivers = ['liuhui@aerozhonghuan.com']

    messages = '<html>各位好，<br />告警信息如下：<br/>'
    messages += '<table border="1" style="border-collapse:collapse;text-align:center;">'
    for ruleInfo in ruleInfoAll:
        compareTitle = ''
        compareValue = ''
        if ruleInfo['check_type'] == '1':
            checkTypeTitle = '期望值'
            checkTypeValue = ruleInfo['excepted_value']
        else:
            checkTypeTitle = '波动值'
            checkTypeValue = ruleInfo['percent_min'] + '%'

        if ruleInfo['rule_type'] == '1':
            if ruleInfo['check_type'] == '1' :
                messages += '<tr><td>规则名称</td><td>规则类型</td><td>数据库名</td><td>表名</td><td>校验类型</td><td>校验方式</td><td>比较方式</td><td>'+checkTypeTitle+'</td><td>今日数据</td></tr>'
                messages += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \
                (ruleInfo['rule_name'],getRuleTypeName(ruleInfo['rule_type']),ruleInfo['db_name'],ruleInfo['table_name'],getCheckTypeName(ruleInfo['check_type']),getCheckModeName(ruleInfo['check_mode']),getCompareModeName(ruleInfo['compare_mode']),checkTypeValue,ruleInfo['result_run'])
            else:
                messages += '<tr><td>规则名称</td><td>规则类型</td><td>数据库名</td><td>表名</td><td>校验类型</td><td>校验方式</td><td>比较方式</td><td>'+checkTypeTitle+'</td><td>今日数据</td><td>对比数据</td></tr>'
                messages += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \
                (ruleInfo['rule_name'],getRuleTypeName(ruleInfo['rule_type']),ruleInfo['db_name'],ruleInfo['table_name'],getCheckTypeName(ruleInfo['check_type']),getCheckModeName(ruleInfo['check_mode']),getCompareModeName(ruleInfo['compare_mode']),checkTypeValue,ruleInfo['result_run'],ruleInfo['compare_num'])
        elif ruleInfo['rule_type'] == '2':
            if ruleInfo['check_type'] == '1':
                messages += '<tr><td>规则名称</td><td>规则类型</td><td>数据库名</td><td>表名</td><td>字段名</td><td>校验类型</td><td>校验方式</td><td>比较方式</td><td>'+checkTypeTitle+'</td><td>今日数据</td></tr>'
                messages += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \
                (ruleInfo['rule_name'],getRuleTypeName(ruleInfo['rule_type']),ruleInfo['db_name'],ruleInfo['table_name'],ruleInfo['column_name'],getCheckTypeName(ruleInfo['check_type']),getCheckModeName(ruleInfo['check_mode']),getCompareModeName(ruleInfo['compare_mode']),checkTypeValue,ruleInfo['result_run'])
            else:
                messages += '<tr><td>规则名称</td><td>规则类型</td><td>数据库名</td><td>表名</td><td>字段名</td><td>校验类型</td><td>校验方式</td><td>比较方式</td><td>'+checkTypeTitle+'</td><td>今日数据</td><td>对比数据</td></tr>'
                messages += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \
                (ruleInfo['rule_name'],getRuleTypeName(ruleInfo['rule_type']),ruleInfo['db_name'],ruleInfo['table_name'],ruleInfo['column_name'],getCheckTypeName(ruleInfo['check_type']),getCheckModeName(ruleInfo['check_mode']),getCompareModeName(ruleInfo['compare_mode']),checkTypeValue,ruleInfo['result_run'],ruleInfo['compare_num'])
        else:
            if ruleInfo['check_type'] == '1':
                messages += '<tr><td>规则名称</td><td>规则类型</td><td>SQL</td><td>校验类型</td><td>校验方式</td><td>比较方式</td><td>'+checkTypeTitle+'</td><td>今日数据</td></tr>'
                messages += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \
                (ruleInfo['rule_name'],getRuleTypeName(ruleInfo['rule_type']),ruleInfo['sql'], getCheckTypeName(ruleInfo['check_type']), getCheckModeName(ruleInfo['check_mode']),
                         getCompareModeName(ruleInfo['compare_mode']), checkTypeValue, ruleInfo['result_run'])
            else:
                messages += '<tr><td>规则名称</td><td>规则类型</td><td>SQL</td><td>校验类型</td><td>校验方式</td><td>比较方式</td><td>'+checkTypeTitle+'</td><td>今日数据</td><td>对比数据</td></tr>'
                messages += '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % \
                (ruleInfo['rule_name'],getRuleTypeName(ruleInfo['rule_type']),ruleInfo['sql'], getCheckTypeName(ruleInfo['check_type']), getCheckModeName(ruleInfo['check_mode']),
                         getCompareModeName(ruleInfo['compare_mode']), checkTypeValue, ruleInfo['result_run'],ruleInfo['compare_num'])


    messages += '</table></html>'

    sendmail(mail,subject,receivers,messages)

def insertExecuteNum(rule_id,result_num):
    """
    插入执行结果条数
    """
    current_day = (datetime.datetime.now()-datetime.timedelta(days=1)).strftime('%Y-%m-%d')
    sql ='''
    delete from hive_rule_execute_num where rule_id='%s' and calculate_date='%s'
    ''' % (rule_id,current_day)
    db_colud.delete(sql)

    insertDict = {
        'rule_id' : rule_id,
        'calculate_date' : current_day,
        'result_num' : result_num,
        'create_time' : datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    }
    db_colud.insert('hive_rule_execute_num',insertDict)


def executeRule():
    """
    执行sql并写入结果
    """
    sql = '''
        select log.id, log.rule_id,log.execute_sql,r.rule_type,r.rule_name,r.custom_sql,r.db_name,r.table_name,r.column_name,
         r.check_type,r.check_mode,r.compare_mode,r.excepted_value,r.percent_min,r.warning_mode,r.warning_receiver,r.sampling_mode,r.filter_condition 
         from hive_rule_execute_log log
         left join hive_quality_rule r on log.rule_id=r.id
         where log.status=0
    '''
    allSql = db_colud.queryAll(sql)

    warningList = []
    for sql in allSql:

        ruleDict = {}

        whereStr = "id=%s" % (sql['id'])
        #判断execute_sql是否已存在，如果是页面提交的数据会没有sql,需要再获取一次
        if sql['execute_sql'] == '':
            quality = common_quality(rule_type=sql['rule_type'], db_name=sql['db_name'],table_name=sql['table_name'], \
                                     column_name=sql['column_name'], custom_sql=sql['custom_sql'],sampling_mode=sql['sampling_mode'], filter_condition=sql['filter_condition'])
            executeSql = quality.get_execute_sql()
            db_colud.update('hive_rule_execute_log', {'execute_sql':executeSql}, whereStr)
            sql['execute_sql'] = executeSql

        startTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        updateDict = {
            'status': 1,
            'start_time': startTime,
        }
        startSeconds = datetime.datetime.strptime(startTime, "%Y-%m-%d %H:%M:%S")


        db_colud.update('hive_rule_execute_log',updateDict,whereStr)
        result_run = 0
        try:
            result_run = execute_sql(sql['execute_sql'])
        except Exception as e:
            error_msg = 'sql:%s 执行报错' % (sql['execute_sql'])
            print("id-%s,rule_id-%s,执行数据-%s"%(sql['id'],sql['rule_id'],error_msg))

            updateDict = {
                'status' : 4,
                'end_time' : datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                'error_msg' : error_msg
            }

            db_colud.update('hive_rule_execute_log', updateDict, whereStr)
            print(result_run)

        insertExecuteNum(sql['rule_id'],result_run)

        #计算执行时间
        endTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        endSeconds = datetime.datetime.strptime(endTime, "%Y-%m-%d %H:%M:%S")
        diffSeconds = (endSeconds-startSeconds).seconds

        updateDict={
            'status' : 2,
            'end_time' : endTime,
            'execute_seconds' : diffSeconds
        }
        db_colud.update('hive_rule_execute_log',updateDict,whereStr)

        #结果值比较
        if sql['check_type'] == '1':
            #执行表达式判断
            compareModeStr = getCompareMode(sql['compare_mode'])
            parse_if = '''if %s %s %s:
    print('success')
else:
    print('error')
              ''' % (result_run, compareModeStr, sql['excepted_value'])
            print(parse_if)
            with stdoutIO() as s:
                exec (parse_if)

            conditionResult = s.getvalue()

            if 'success' in conditionResult:
                try:
                    sendWarningMail(sql,result_run)
                except Exception as e:
                    print('id-%s,rule_id-%s 发送邮件失败' % (sql['id'],sql['rule_id']))

                ruleDict = sql
                ruleDict['result_run'] = result_run
                ruleDict['compare_num'] = 0
                warningList.append(ruleDict)

                db_colud.update('hive_rule_execute_log', {'status':3}, whereStr)

                print('触发了警告：id-%s,rule_id-%s,result_rum-%s,excepted_value-%s' % (sql['id'],sql['rule_id'],result_run,sql['excepted_value']))
            else:
                print('正常执行：id-%s,rule_id-%s,result_rum-%s,excepted_value-%s' % (sql['id'],sql['rule_id'],result_run,sql['excepted_value']))

        elif sql['check_type'] == '2':
            resultNumDict = getCompareData(sql['rule_id'],sql['check_mode'])
            resultNum = int(resultNumDict['result_num'])
            #print(resultNum)

            if resultNum >0:
                print('正常执行：id-%s,rule_id-%s,result_run-%s,percent_num-%s' % (sql['id'],sql['rule_id'],result_run,resultNum))

                if sql['compare_mode'] in ['rise','abs']:
                    percentNum = resultNum*(1+(float(sql['percent_min'])/100))

                    if result_run > percentNum:
                        # 发送邮件
                        try:
                            sendWarningMail(sql, result_run,resultNum)
                        except Exception as e:
                            print('id-%s,rule_id-%s 发送邮件失败' % (sql['id'], sql['rule_id']))

                        ruleDict = sql
                        ruleDict['result_run'] = result_run
                        ruleDict['compare_num'] = resultNum
                        warningList.append(ruleDict)

                        db_colud.update('hive_rule_execute_log', {'status': 3}, whereStr)
                        print('触发了上涨的警告：id-%s,rule_id-%s,result_run-%s,resultNum-%s,percent_num-%s' % (sql['id'], sql['rule_id'], result_run,resultNum, percentNum))

                if sql['compare_mode'] in ['decline','abs']:
                    percentNum = resultNum*(1-(float(sql['percent_min'])/100))
                    if result_run < percentNum:
                        #发送邮件
                        try:
                            sendWarningMail(sql, result_run,resultNum)
                        except Exception as e:
                            print('id-%s,rule_id-%s 发送邮件失败' % (sql['id'], sql['rule_id']))

                        ruleDict = sql
                        ruleDict['result_run'] = result_run
                        ruleDict['compare_num'] = resultNum
                        warningList.append(ruleDict)

                        db_colud.update('hive_rule_execute_log', {'status': 3}, whereStr)
                        print('触发了下降的警告：id-%s,rule_id-%s,result_run-%s,resultNum-%s,percent_num-%s' % (sql['id'], sql['rule_id'], result_run,resultNum, percentNum))

            else:
                print('历史数据没有获取到，不触发规则：id-%s,rule_id-%s,result_run-%s,percent_num-%s' % (sql['id'],sql['rule_id'],result_run,resultNum))

    #存在出发报警的规则一起发送，不单独发送了
    if warningList:
        pass
        #sendWarningMailAll(warningList)

# 查找指定目录下的文件
def find_file(search_path, include_str=None, filter_strs=None):
    """
    查找指定目录下所有的文件（不包含以__开头和结尾的文件）或指定格式的文件，若不同目录存在相同文件名，只返回第1个文件的路径
    :param search_path: 查找的目录路径
    :param include_str: 获取包含字符串的名称
    :param filter_strs: 过滤包含字符串的名称
    """
    if filter_strs is None:
        filter_strs = []

    files = []
    # 获取路径下所有文件
    names = os.listdir(search_path)
    for name in names:
        path = os.path.abspath(os.path.join(search_path, name))
        if os.path.isfile(path):
            # 如果不包含指定字符串则
            if include_str is not None and include_str not in name:
                continue

            # 如果未break，说明不包含filter_strs中的字符
            for filter_str in filter_strs:
                if filter_str in name:
                    break
            else:
                files.append(name)
        else:
            pass
            #files += find_file(path, include_str=include_str, filter_strs=filter_strs)
    return files

# 插入makefile 调度完成的表
def insertFinishTable():
    filePath = '/project/status/'

    # 当天已经跑完的
    executeDate = (datetime.datetime.now()-datetime.timedelta(days=1)).strftime('%Y-%m-%d')
    filePathNew = filePath + executeDate
    fileCurrentList = find_file(filePathNew, filter_strs=[".log", ".start",".xls"])

    # 当月已经跑完的
    executeMonthDate = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime('%Y-%m-01')
    filePathMonth = filePath + executeDate
    fileMonthList = find_file(filePathMonth, filter_strs=[".log", ".start", ".xls"])

    fileCurrentList.extend(fileMonthList)
    fileList = list(set(fileCurrentList))
    #print(fileList)

    for table in fileList:
        # 判断当天的数据是否在表里存在
        sql = ''' select id from hive_schedule_status 
                where schedule_type='makefile' and execute_date='%s' and table_name='%s' ''' % (executeDate,table)
        statusInfo = db_colud.queryRow(sql)
        if statusInfo:
            pass
        else:
            insertDict = {
                'schedule_type' : 'makefile',
                'table_name' : table.lower(),
                'status' : 'SUCCEEDED',
                'execute_date' : executeDate,
                'create_time' : datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            }

            db_colud.insert('hive_schedule_status',insertDict)
            print('schedule_type:makefile,table:%s insert' % (table))


def addCustomRuleToExecute():
    """
    添加自定义执行规则到执行计划里面
    :return:
    """
    executeDate = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
    executeTime = datetime.datetime.now().strftime('%H:%M')
    ruleSql = '''
    select r.* from hive_quality_rule r 
    inner join hive_schedule_status s on r.table_name=s.table_name
    where r.status='on' and r.execute_type=2 and s.status='SUCCEEDED' and s.execute_date='%s'
    ''' % (executeDate)
    ruleAll = db_colud.queryAll(ruleSql)

    if ruleAll:
        for rule in ruleAll:
            execute_hour = rule['execute_hour']

            #print('execute_hour:%s,execute_time:%s' %(execute_hour,executeTime))
            if executeTime>= execute_hour:
                print('rule_id:%s 自定义执行加入队列中' % (rule['id']))
                insertRuleToTable(rule)
            else:
                print('rule_id:%s 时间判断不符合,暂不加入'% (rule['id']))

    # 还有一种情况，系统统一执行时依赖没有生成需要等待依赖生成再加入执行列表
    if executeTime > '14:00':
        ruleSql = '''
            select r.* from hive_quality_rule r 
            inner join hive_schedule_status s on r.table_name=s.table_name
            where r.status='on' and r.execute_type=1 and s.status='SUCCEEDED' and s.execute_date='%s'
            ''' % (executeDate)
        ruleAll = db_colud.queryAll(ruleSql)
        if ruleAll:
            for rule in ruleAll:
                # print('execute_hour:%s,execute_time:%s' %(execute_hour,executeTime))
                print('rule_id:%s 统一执行依赖没有生成的rule,自动加入队列中' % (rule['id']))
                insertRuleToTable(rule)



if __name__ == '__main__':

    db_colud = getDbCloud()

    if (len(sys.argv) > 1):
        act = sys.argv[1]
    else:
        act = 'monitor'

    #任务监控，每10分钟执行一次，满足执行任务的都放入待执行列表中
    if act == 'monitor':
        addRuleToExecute()
    else:
        #insertFinishTable()
        addCustomRuleToExecute()
        executeRule()

